package edu.unc.ils.harvest;

import java.io.BufferedReader;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import javax.xml.parsers.DocumentBuilder;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.xpath.XPath;
import javax.xml.xpath.XPathConstants;
import javax.xml.xpath.XPathFactory;

import org.apache.commons.net.ftp.FTP;
import org.apache.commons.net.ftp.FTPClient;
import org.apache.commons.net.ftp.FTPReply;
import org.w3c.dom.Document;
import org.w3c.dom.Node;
import org.w3c.dom.NodeList;

public class NLMHarvester 
{
	public void getArticles() throws Exception
	{
		BufferedReader br = new BufferedReader(new FileReader(
				new File("/Users/cwillis/dev/mrc/dryad_hive/journal_analysis/pubmed/agricola_pubmed_articles.txt")));
		String line;
		Map<String, String> agricolaPmidMap = new HashMap<String, String>();
		List<String> pmids = new ArrayList<String>();
		while ((line = br.readLine()) != null)
		{
			String[] fields = line.split("\\|");
			String agricolaId = fields[0];
			String pmid = fields[1];
			pmids.add(pmid);
			agricolaPmidMap.put(pmid, agricolaId);
		}
		
		br = new BufferedReader(new FileReader(
				new File("/Users/cwillis/dev/mrc/dryad_hive/journal_analysis/pubmed/PMC-ids.csv")));
		Map<String, String> pmidPMCIdMap = new HashMap<String, String>();
		while ((line = br.readLine()) != null)
		{
			String[] fields = line.split(",");
			String pmcid = fields[8];
			String pmid = fields[9];
			if (pmids.contains(pmid))
				pmidPMCIdMap.put(pmcid, pmid);
		}
		
		br = new BufferedReader(new FileReader(
				new File("/Users/cwillis/dev/mrc/workspace/losee//output/NLM/articles/file_list.csv")));
		int i = 0;
		while ((line = br.readLine()) != null)
		{
			if (i==0) {
				i++;
				continue;
			}
			String[] fields = line.split("\\,");
			String file = fields[0];
			String pmcid = fields[2];
			String pmid = pmidPMCIdMap.get(pmcid);
			String agricolaId = agricolaPmidMap.get(pmid);
			if (agricolaId != null)
			{
				getFile(file, agricolaId, pmcid);
			}
			i++;
		}
	}
	
	public void getFile(String path, String pmid, String pmcid)
	{

		String baseDir = "/Users/cwillis/Desktop/gold_set/agricola";
		
		FTPClient ftp = new FTPClient();
		try
		{
			//ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/file_list.csv
			ftp.connect("ftp.ncbi.nlm.nih.gov");
			int reply = ftp.getReplyCode();
			if (FTPReply.isPositiveCompletion(reply))	
			{
				ftp.login("anonymous", "craig.willis@unc.edu");
				ftp.enterLocalPassiveMode();
				
				ftp.setFileType(FTP.BINARY_FILE_TYPE);
				System.out.println(ftp.getReplyString());
				
				File dir = new File(baseDir + File.separator + pmid);
				dir.mkdirs();
				
				File file = new File(dir.getAbsolutePath() + File.separator + pmid + ".tar.gz");
				
				FileOutputStream fos = new FileOutputStream(file);
				ftp.retrieveFile("/pub/pmc/" + path, fos);
				System.out.println(ftp.getReplyString());
			}
			ftp.disconnect();
		} catch (Exception e) {
			e.printStackTrace();
		}
	}

	// http://www.ncbi.nlm.nih.gov/pmc/tools/ftp/
	public void getFullText() throws Exception
	{
		
		Map<String, String> journalMap = getJournals();
		
		FTPClient ftp = new FTPClient();
		try
		{
			//ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/file_list.csv
			ftp.connect("ftp.ncbi.nlm.nih.gov");
			int reply = ftp.getReplyCode();
			if (FTPReply.isPositiveCompletion(reply))	
			{
				ftp.login("anonymous", "craig.willis@unc.edu");
				ftp.enterLocalPassiveMode();

				
				File fileList = new File("output" + File.separator + "NLM" + 
						File.separator + "articles" + File.separator + "file_list.csv");
				Map<String, String> fileMap = new HashMap<String, String>();
				if (!fileList.exists())
				{
					InputStream is = ftp.retrieveFileStream("/pub/pmc/file_list.csv");
					if (is == null)
						System.out.println(ftp.getReplyString());
					BufferedReader br = new BufferedReader(new InputStreamReader(is));
					FileWriter fw = new FileWriter(fileList);
					String line;
					while((line = br.readLine()) != null) {
						fw.write(line + "\n");
						String[] fields = line.split(",");
						String path = fields[0];
						String pmcid = fields[2];
						fileMap.put(pmcid, path);
					}
					br.close();
					fw.close();
					is.close();
				} 
				else
				{
					BufferedReader br = new BufferedReader(new FileReader(fileList));
					String line;
					while((line = br.readLine()) != null) {
						String[] fields = line.split(",");
						String path = fields[0];
						String pmcid = fields[2];
						fileMap.put(pmcid, path);
					}
					br.close();
				}
			
				ftp.setFileType(FTP.BINARY_FILE_TYPE);
				System.out.println(ftp.getReplyString());
				BufferedReader br = new BufferedReader(new FileReader("/Users/cwillis/dev/mrc/journal_analysis/open_access_pubmed_pmcids.csv"));
				String line;
				while((line = br.readLine()) != null)
				{
					String fields[] = line.split(",");
					String issn = fields[1];
					String essn = fields[2];
					String pmcid = fields[8];
					String pmid = fields[9];
					
					if (pmid.equals("0"))
						continue;
					
					String path = fileMap.get(pmcid);
					if (path == null)
						continue;
					
					String fileName = path.substring(path.lastIndexOf("/") + 1, path.length());
					
					if (journalMap.get(essn) != null)
						issn = essn;
					
					File dir = new File("output" + File.separator + "NLM" + 
							File.separator + "articles" + File.separator + issn + File.separator + pmid + File.separator);
					if (!dir.exists())
						dir.mkdirs();
					File file = new File( dir.getAbsolutePath() + File.separator + fileName);
					
					if (!file.exists())
					{
						System.out.println("Getting file:" + file);
						FileOutputStream fos = new FileOutputStream(file);
						ftp.retrieveFile("/pub/pmc/" + path, fos);
						System.out.println(ftp.getReplyString());
					}
					else
						System.out.println(file + " exists");

				}
			}
			ftp.disconnect();
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	public void getArticleInfo() throws Exception
	{
		FileWriter documents = new FileWriter("output" + File.separator + "NLM" + File.separator + "pubmed_articles.txt");
		FileWriter subjects = new FileWriter("output" + File.separator + "NLM" + File.separator + "pubmed_subjects.txt");
		// ISSN, Title
		Map<String, String> journals = getJournals(); // new HashMap<String,String>();//
		//journals.put("1976-6696", "BMB reports");
		
		for (String issn: journals.keySet())
		{
			File issnDir = new File("output" + File.separator + "NLM" + 
				File.separator + "articles" + File.separator + issn);

			File[] records = issnDir.listFiles();
			if (records == null)
				continue;
			
			for (File record: records)
			{
	            DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
	            DocumentBuilder db = dbf.newDocumentBuilder();
	            Document doc = db.parse(new FileInputStream(record));
			    XPathFactory  factory=XPathFactory.newInstance();
			    XPath xPath=factory.newXPath();
			    
			    NodeList nodeList = (NodeList)xPath.evaluate("/PubmedArticleSet/PubmedArticle/MedlineCitation/PMID",
			            doc, XPathConstants.NODESET);
			   
			    String pmid = "";
			    for (int j=0; j<nodeList.getLength(); j++)
			    {
		            Node node = nodeList.item(j);
		            pmid = node.getNodeValue();
			    }
			    
			    nodeList = (NodeList)xPath.evaluate("/PubmedArticleSet/PubmedArticle/MedlineCitation/Article/Journal/Title",
			            doc, XPathConstants.NODESET);
			   
			    String journalTitle = "";
			    for (int j=0; j<nodeList.getLength(); j++)
			    {
		            Node node = nodeList.item(j);
		            journalTitle = node.getNodeValue();
			    }
			    
			    nodeList = (NodeList)xPath.evaluate("/PubmedArticleSet/PubmedArticle/MedlineCitation/Article/ArticleTitle",
			            doc, XPathConstants.NODESET);
			   
			    String articleTitle = "";
			    for (int j=0; j<nodeList.getLength(); j++)
			    {
		            Node node = nodeList.item(j);
		            articleTitle = node.getNodeValue();
			    }
			    
			    nodeList = (NodeList)xPath.evaluate("/PubmedArticleSet/PubmedArticle/MedlineCitation/Article/Abstract/AbstractText",
			            doc, XPathConstants.NODESET);
			   
			    String abs = "";
			    for (int j=0; j<nodeList.getLength(); j++)
			    {
		            Node node = nodeList.item(j);
		            abs += node.getNodeValue();
			    }
			    
			    
			    nodeList = (NodeList)xPath.evaluate("/PubmedArticleSet/PubmedArticle/MedlineCitation/MeshHeadingList/MeshHeading",
			            doc, XPathConstants.NODESET);
			    documents.write(pmid + "|" + journalTitle + "|" + articleTitle + "\n");
			    for (int j=0; j<nodeList.getLength(); j++)
			    {
		            Node node = nodeList.item(j);
		            NodeList children = node.getChildNodes();
		            String descriptor = "";
		            String descMajorTopic = "";
		            List<String[]> qualifiers = new ArrayList<String[]>();
				    for (int k=0; k<children.getLength(); k++)
				    {
				    	Node child = children.item(k);
				    	String name = child.getNodeName();
				    	if (name.equals("DescriptorName")) {
				    		descriptor = child.getNodeValue();
				    		descMajorTopic = child.getAttributes().getNamedItem("MajorTopicYN").getNodeValue();
				    	}
				    	else if (name.equals("QualifierName")) {
				    		String qualifier = child.getNodeValue();
				    		String qualMajorTopic = child.getAttributes().getNamedItem("MajorTopicYN").getNodeValue();
				    		String[] fields =  new String[]{qualifier, qualMajorTopic};
				    		if (!qualifiers.contains(fields))
				    			qualifiers.add(fields);
				    	}
				    }
				    
				    if (qualifiers.size()>0)
				    {
				    	for (String[] fields: qualifiers)
				    		subjects.write(pmid + "|" + j +"|" + descriptor + "|" + descMajorTopic + "|" + fields[0] + "|" + fields[1] + "\n");
				    }
				    else
				    	subjects.write(pmid + "|"  + j +"|" + descriptor + "|" + descMajorTopic + "||" + "\n");
			    }
			    documents.flush();
			    subjects.flush();

			}


		}
		documents.close();
		subjects.close();
	}
	
	public void getArticleRecords() throws Exception
	{
		Map<String, String> journalMap = getJournals();
		
		BufferedReader br = new BufferedReader(new FileReader("/Users/cwillis/dev/mrc/journal_analysis/open_access_pubmed_pmcids.csv"));
		String line;
		
		int c = 0;
		while ((line = br.readLine()) != null) 
		{
			c++;
			System.out.println(c);
			Thread.sleep(400);
			String[] fields = line.split(",");
			String issn = fields[1];
			String essn = fields[2];
			
			String pmid = fields[9];
			try
			{
				URL u = new URL ("http://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id=" + pmid + "&retmode=xml&email=craig.willis%40unc.edu");
				InputStream is = u.openStream();
			
				if (journalMap.get(essn) != null)
					issn = essn;
				
				File dir = new File("output" + File.separator + "NLM" + 
						File.separator + "articles" + File.separator + issn + File.separator);
				if (!dir.exists())
					dir.mkdirs();
				File file = new File( dir.getAbsolutePath() + File.separator + pmid + ".xml");
				FileWriter fw = new FileWriter(file);
				int i;
				while ((i = is.read()) != -1)
					fw.write(i);
				fw.close();
				is.close();			
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
	}
	
	
	public Map<String, String> getJournals() throws Exception
	{
		BufferedReader br = new BufferedReader(new FileReader("/Users/cwillis/dev/mrc/journal_analysis/open_access_pubmed_journals.txt"));
		String line;
		Map<String, String> journalMap = new HashMap<String, String>();
		while ((line = br.readLine()) != null) 
		{
			String[] fields = line.split("\\t");
			String title = fields[0];
			String issn = fields[1];
			journalMap.put(issn, title);
		}
		return journalMap;
	}
	public void getSerialsInfo() throws Exception
	{
		Map<String, String> journals = getJournals();
		
		for (String issn: journals.keySet())
		{
			File issnDir = new File("output" + File.separator + "NLM" + 
				File.separator + "serials" + File.separator + issn);

			File[] records = issnDir.listFiles();
			if (records == null)
				continue;
			
			for (File record: records)
			{
	            DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();
	            DocumentBuilder db = dbf.newDocumentBuilder();
	            Document doc = db.parse(new FileInputStream(record));
			    XPathFactory  factory=XPathFactory.newInstance();
			    XPath xPath=factory.newXPath();
			    
			    
			    // 245 $a = title
			    NodeList nodeList = (NodeList)xPath.evaluate("/collection/record/datafield[@tag='245']/subfield[@code='a']",
			            doc, XPathConstants.NODESET);
			   
			    String title = "";
			    for (int j=0; j<nodeList.getLength(); j++)
			    {
			            Node node = nodeList.item(j);
			            title = node.getNodeValue();
			    }
			    
			    // $650 $a
			    nodeList = (NodeList)xPath.evaluate("/collection/record/datafield[@tag='650']/subfield[@code='a']",
			            doc, XPathConstants.NODESET);
			   
			    List<String> subjects = new ArrayList<String>();;
			    for (int j=0; j<nodeList.getLength(); j++)
			    {
			            Node node = nodeList.item(j);
			            String subject = node.getNodeValue();
			            if (!subjects.contains(subject))
			            	subjects.add(subject);
			    }
			    
			    System.out.println(issn + "|" + title + "|" + subjects);
			}
		}
	}
	public void getSerialRecords()
	{
		try
		{
			Map<String, String> journalMap = getJournals();

			Z3950Client ah = new Z3950Client();
			ah.connect("tegument.nlm.nih.gov", 7091);
			// ISSN  8,3
			// Title 4

			for (String issn: journalMap.keySet())
			{

				String title = journalMap.get(issn);
				
				System.out.println(issn + ": " + title);
				ah.setOutputDirectory(new File("output" + File.separator + "NLM" + File.separator + issn));
				ah.search("@attrset bib-1 @attr 1=8 \"" + issn + "\"", title, issn);
			}
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	
	public static void main(String[] args) throws Exception {
		NLMHarvester h = new NLMHarvester();
		//h.getArticleRecords();
		//h.getArticleInfo();
		//h.getFullText();
		h.getArticles();
	}
}
